package com.ruiyin.service;


import java.util.List;
import com.ruiyin.utils.SpiderUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;

@Service
public class SpiderService extends SpiderUtils implements PageProcessor {

    @Override
    public void process(Page page) {
        Html html = page.getHtml();

        //获取图片
        String imgUrl = html.css("div.main-image p a img", "src").get();
        if (imgUrl != null) {
            String dirs = html.xpath("div[@class=currentpath]/text()").get();
            String[] dir = dirs.split(" » ");

            //下载图片
            downImg(imgUrl,dir[1]+dir[2]);

            //内容页的下一页
            String next = html.css("div.pagenavi a:last-child").links().get();
            page.addTargetRequest(next);


        } else {

            //封面页链接
            List<String> conUrl = html.css("ul#pins li span a").links().all();
            page.addTargetRequests(conUrl);

            //下一页链接
            String nextPage = html.css("div.nav-links a.next").links().get();
            page.addTargetRequest(nextPage);

        }
    }


    private Site site = Site.me()
            .setCharset("utf8")//设置编码
            .setTimeOut(30*1000)//设置超时
            .setRetrySleepTime(3000)//间隔多久重试
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36")
            .addHeader("authority","www.mzitu.com")
            .addHeader("path","/xinggan/")
            .addHeader("accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3")
            .addHeader("cookie","Hm_lvt_dbc355aef238b6c32b43eacbbf161c3c=1575464236; Hm_lpvt_dbc355aef238b6c32b43eacbbf161c3c=1575465527")
            .setRetryTimes(3);//重试次数

    @Override
    public Site getSite() {
        return site;
    }
    private static final int THREAD_NUM = 10;

    private String[] url =
            {
                    "https://www.mzitu.com/xinggan/",
                    "https://www.mzitu.com/japan/",
                    "https://www.mzitu.com/taiwan/",
                    "https://www.mzitu.com/mm/",
                    "https://www.mzitu.com/zipai/",
                    "https://www.mzitu.com/jiepai/"
            };

    @Scheduled(initialDelay = 1000,fixedDelay = 190000000)
    public void Main(){
        long start,end;
        System.err.println("开始爬取图片...");
        start = System.currentTimeMillis();
        Spider.create(new SpiderService())
                .thread(THREAD_NUM)
                .addUrl(url)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(50000)))//使用内存操作
                .run();
        end = System.currentTimeMillis();
        System.err.println("图片爬取完成...\n共计时："+(end-start)/1000+"秒"+"为："+((end - start)/1000)/60+"分");
    }
}
